import json
import datetime
import os
from collections import Counter
import csv
ht=[ '#LGBTQHistory', '#HorrorToK', '#ProblemSolved', '#SkateLife', '#ThriftShop', '#SpookyTreats',  '#tiktokfood', '#interiordesign', '#sfxmakeup', '#ComingOfAge', '#falldiy', '#worldseries', '#gaminglife', '#meleaving', '#yellow', '#yougotthis','#mycostume','#mypfp','#welldone','#nativefamily','#happyhalloween', '#OhNo','#homeoffice','#myrecommendation','#myhobby','#fanedit','#stemlife','#motivationmonday','#wip','#veteransday','#holidaycountdown','#onhold','#bekind','#WeekendVibes','#growupwithme','#holidaysourway','#tabletop','#RoomTour','#bakingszn','#nbadraft',
     '#nonuancenovember','#theatrekids','#gamingsetup','#carsoftiktok','#needtoknow','#whenwewereyounger','#halloweenishere','#ourtype','#fallguysmoments','#workingathome','#onlinedating','#familyrecipe','#givingthanks','#readysetshop','#diceroll','#holidayvibes','#coldweather','#holidaytiktok','#recordsday','#inkdrawing','#happyholidays','#homecooked','#rnbvibes','#easydiy','#wildanimals','#neonshadow','graphicdesign' , '#youwantmore' ,'#familyimpression','#givingszn','#watchmegrow','#whereilive','foodtiktok','#selfimprovement','#holidaymusic','#howbizarre','#ImAGhost', '#personalfinance']
datalist={}
for hashtag in ht:
    datalist[hashtag]=[]
    for name in os.listdir('D:\\Work\\Tool\\tiktok\\TikToks\\'):
        if hashtag.lower() in name.lower():
            datalist[hashtag].append(name)

with open('D:\\Work\\Tool\\tiktok\\compare_days.tsv', 'w', encoding='utf-8', newline='\n') as filename_output:
    writer = csv.writer(filename_output, delimiter='\t')
    writer.writerow(['filename','#unique ids','#in previous not in this','#in this not in previous'])
            
    for hashtag in ht:
        c=Counter()
        print(hashtag)
        ids={}
        uids=[]
        uidscount=[]
        for fn in datalist[hashtag]:
            ids[fn]=[]
            with open('D:\\Work\\Tool\\tiktok\\TikToks\\' + fn, 'r', encoding='utf-8', newline='\n') as filename_input:
                for line in filename_input:
                    z = json.loads(line)
                    lid=''
                    if 'id' in z.keys():
                        lid=z['id']
                        
                    elif 'itemInfos' in z.keys():
                        lid=z['itemInfos']['id']
                    if lid not in uids:
                        uids.append(lid)
                    ids[fn].append(lid)
                uidscount.append(len(uids))
        writer.writerow([datalist[hashtag][0],uidscount[0],'NA','NA'])
        for i in range(len(datalist[hashtag])-1):
            ipnit=[item for item in ids[datalist[hashtag][i]] if item not in ids[datalist[hashtag][i+1]]]
            itnip=[item for item in ids[datalist[hashtag][i+1]] if item not in ids[datalist[hashtag][i]]]
            writer.writerow([datalist[hashtag][i+1], uidscount[i+1],len(ipnit),len(itnip)])
                
        
